package cn.net.withub.dataCollector.web.collector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.net.withub.dataCollector.common.editor.DateUtil;
import cn.net.withub.dataCollector.common.model.*;
import cn.net.withub.dataCollector.common.utils.CreateNewKey;
import cn.net.withub.dataCollector.common.utils.HTMLSpirit;
import cn.net.withub.dataCollector.web.service.CollectorService;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.context.WebApplicationContext;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

/**
 * Created by hupeng on 2018/6/8.
 */
public class CollectorUtilsM44 extends BreadthCrawler {

    //@Resource
    private CollectorService collectorService;

    //@Resource
    private WebApplicationContext webAppliction;

    private TCollectorConfig tCollectorConfig;

    private Logger log = Logger.getLogger(this.getClass());

    public CollectorUtilsM44(String crawlPath, boolean autoParse, WebApplicationContext wac, TCollectorConfig config) {
        super(crawlPath, autoParse);

        webAppliction = wac;
        if (webAppliction != null) {
            collectorService = (CollectorService) webAppliction.getBean("collectorService");
        }
        if (config != null) {
            tCollectorConfig = config;
        }
    }

    public static void main(String[] args) throws Exception {
        CollectorUtilsM20 crawler = new CollectorUtilsM20("crawl", true, null, null);
        crawler.setThreads(10);
        crawler.setTopN(100);
        crawler.setResumable(true);
        crawler.start(500);

        /*Executor executor=new Executor() {
            public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
                HtmlUnitDriver driver = new HtmlUnitDriver();
                driver.setJavascriptEnabled(true);
                driver.get(datum.getUrl());
                //WebElement element=driver.findElementByCssSelector("ul#ssxw");
                WebElement element=driver.findElementByCssSelector("span#outlink1");
                log.info("反链数:"+element.getText());
            }
        };

        //创建一个基于伯克利DB的DBManager
        DBManager manager=new BerkeleyDBManager("crawl");
        //创建一个Crawler需要有DBManager和Executor
        Crawler crawler= new Crawler(manager,executor);
        //crawler.addSeed("http://149.0.160.10");
        crawler.addSeed("http://seo.chinaz.com/?host=www.tuicool.com");
        crawler.start(1);*/
    }

    public void CollectorUtils(CollectorService collectorService) {
        collectorService = collectorService;
    }

    public void visit(Page page, CrawlDatums crawlDatums) {
        String url = page.getUrl();
        String html = page.getHtml();
        String contentType = page.getResponse().getContentType();
        //log.info("--------------------------URL:\n" + url+"     "+contentType);
        try {
            //http://149.8.0.202/Information/index.asp?classid=97
            if (page.matchUrl("http://.*/Information/infolist.asp\\?classid=.*") || page.matchUrl("http://.*/Information/infolist.asp\\?p=.*&classid=.*")) {
                //log.info("--------------------------html:\n"+page.html());
                //获取新闻列表内容
                Elements trElements = page.select("tr[bgcolor]:has(a[href][target=_top])");
                for (Element tr : trElements) {
                    String fbsj = null;
                    String title = null;
                    String src = null;
                    String bjsh = null;
                    try {
                        //获取发布时间
                        Elements tdElements = tr.select("td[width=10%]");
                        if (tdElements != null && tdElements.size() > 0) {
                            fbsj = tdElements.get(0).text().replaceAll("  ", "").replaceAll(" ", "");
                        }
                        fbsj = fbsj.replace(".", "-");
                        log.info("fbsj...." + fbsj);

                        //获取发布标题
                        Elements titleElements = tr.select("td[width=75%] a[href][target=_top]");
                        if (titleElements != null && titleElements.size() > 0) {
                            title = titleElements.get(0).text();
                        }
                        log.info("title...." + title);

                        //获取作者
                        Elements tbjshElements = tr.select("td[width=15%]");
                        if (tbjshElements != null && tbjshElements.size() > 0) {
                            bjsh = tbjshElements.get(0).text().replaceAll("  ", "").replaceAll(" ", "");
                        }
                        log.info("bjsh...." + bjsh);

                        Elements elements = tr.select("a[href][target]");
                        //log.info("-----------href url size:"+elements.size()+"........."+tr.select("td a[href][target]").size());
                        for (Element element : elements) {
                            //a标签 href
                            src = element.attr("abs:href");
                            log.info("href url:\n" + src);
                            //重新添加爬取任务,增量抓取
                            if (tCollectorConfig.getFbsj() == null || (tCollectorConfig.getFbsj() != null && DateUtil.compareDate(fbsj, new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj())) > 0)) {
                                log.info("fbsj: " + fbsj);
                                crawlDatums.add(
                                        new CrawlDatum(src)
                                                .meta("fbsj", fbsj)
                                                .meta("title", title)
                                                .meta("firstUrl", url)
                                                .meta("bjsh", bjsh)
                                );
                            }
                            //判断已抓取过的最大发布日期，根据标题和发布时间进行对比，未抓取过的进行抓取
                            else if (tCollectorConfig.getFbsj() != null && DateUtil.compareDate(fbsj, new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj())) == 0) {
                                log.info("fbsj: " + fbsj + "相同");
                                TCollectorData tCollectorData = (TCollectorData) collectorService.load(TCollectorData.class, " fbsj='" + fbsj + "' and title='" + title + "'");
                                if (tCollectorData == null) {
                                    log.info("fbsj: " + fbsj + "抓取");
                                    crawlDatums.add(
                                            new CrawlDatum(src)
                                                    .meta("fbsj", fbsj)
                                                    .meta("title", title)
                                                    .meta("firstUrl", url)
                                                    .meta("bjsh", bjsh)
                                    );
                                }
                            }

                        }
                    } catch (Exception e) {
                        saveLog(title, fbsj, url, src, 0);
                        e.printStackTrace();
                        continue;
                    }
                }

                //获取下一页
                String src = "";
                try {
                    Elements pageElements = page.select("a[class=white_bg]");
                    for (Element element : pageElements) {
                        src = element.attr("abs:href");
                        String text = element.text().trim();
                        if (text.equals("下一页")) {
                            log.info("抓取下一页......" + src);
                            crawlDatums.add(
                                    new CrawlDatum(src)
                            );
                        }
                    }
                } catch (Exception e) {
                    saveLog("", "", url, src, 0);
                    e.printStackTrace();
                }
            } else if (page.matchUrl("http://.*/information/informationDisplay.asp\\?NewsID=.*") || page.matchUrl("http://.*/Information/InformationDisplay.asp\\?NewsID=.*")) {
                //http://149.8.0.202/information/InformationDisplay.asp?NewsID=18935
                log.info("contentType:     \n" + contentType + "     meta:" + page.meta("title"));
                Map<String, Object> map = new HashMap<String, Object>();
                String content = "";
                String serverFilePath = "";
                String wjgs = "";

                String title = page.meta("title");
                String fbsj = page.meta("fbsj").replaceAll("  ", "").replaceAll(" ", "").replace(".", "-");
                String firstUrl = page.meta("firstUrl");
                String bjsh = page.meta("bjsh");
                int zt = 0;

                if (contentType.contains("msword") || contentType.contains("pdf")) {
                    try {
                        //获取word文件
                        if (contentType.contains("msword")) {
                            map = HttpClientUtils.getWord(url);
                            wjgs = "doc";
                        }
                        //获取pdf文件
                        else if (contentType.contains("pdf")) {
                            map = HttpClientUtils.getPdf(url);
                            wjgs = "pdf";
                        }

                        if (map != null) {
                            content = map.get("content") == null ? "" : map.get("content").toString();
                            serverFilePath = map.get("serverFilePath") == null ? "" : map.get("serverFilePath").toString();
                        }

                        Date date = new Date();
                        String id = new CreateNewKey().createId();
                        TCollectorData tCollectorData = new TCollectorData();
                        tCollectorData.setId(id);
                        tCollectorData.setConfigId(tCollectorConfig.getId());
                        tCollectorData.setTitle(title);
                        Calendar nowdate = Calendar.getInstance();
                        String year = String.valueOf(nowdate.get(Calendar.YEAR));
                        if (null != fbsj && !"".equals(fbsj)) {
                            if (Integer.parseInt(fbsj.substring(0, 4)) > Integer.parseInt(year)) {
                                fbsj = year + fbsj.substring(4);
                                //log.info("fbsj:" + fbsj);
                            }
                        }
                        tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        tCollectorData.setWjgs(wjgs);
                        tCollectorData.setUrl(url);
                        tCollectorData.setFydm(tCollectorConfig.getFydm());
                        tCollectorData.setFymc(tCollectorConfig.getFymc());
                        tCollectorData.setWjlj(serverFilePath);
                        tCollectorData.setSystime(date);
                        collectorService.save(tCollectorData);

                        TCollectorContent tCollectorContent = new TCollectorContent();
                        tCollectorContent.setId(new CreateNewKey().createId());
                        tCollectorContent.setContent(content);
                        tCollectorContent.setSystime(date);
                        tCollectorContent.setDataId(id);
                        collectorService.save(tCollectorContent);

                        //保存最大发布时间
                        TCollectorConfig config = (TCollectorConfig) collectorService.load(TCollectorConfig.class, " id='" + tCollectorConfig.getId() + "'");
                        if (config.getFbsj() == null || DateUtil.compareDate(fbsj, new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj())) > 0) {
                            log.info("config........." + fbsj);
                            config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                            collectorService.save(config);
                        }

                        zt = 1;
                    } catch (Exception e) {
                        zt = 0;
                        e.printStackTrace();
                    } finally {
                        saveLog(title, fbsj, firstUrl, url, zt);
                    }
                }
                //html文件
                else if (contentType.contains("html")) {
                    try {
                        //获取发布相关信息 发布时间：2018.04.24  作者:研究室  刘慧  点击次数：103 次
                        //发布时间：2018.04.11  作者:系统管理员  点击次数：108 次
                        String ttt1 = page.select("td[class=bodytd2]").text();
                        int index_fbsj = ttt1.indexOf("发布时间：");
                        int index_bjsh = ttt1.indexOf("作者:");
                        if (index_bjsh <= -1) {
                            index_bjsh = ttt1.indexOf("作者：");
                        }
                        int index_djcs = ttt1.indexOf("点击次数：");

                        if (index_fbsj > -1) {
                            fbsj = ttt1.substring(index_fbsj + 5, index_bjsh).replaceAll(" ", "").replaceAll("  ", "").replace(".", "-");
                        }
                        bjsh = null;
                        if (index_bjsh > -1)
                            bjsh = ttt1.substring(index_bjsh + 3, index_djcs);
                        String djcs = null;
                        if (index_djcs > -1)
                            djcs = ttt1.substring(index_djcs + 5);

                        //获取iframe内容
                        Elements iframe = page.select("iframe[id=ifrName][name=ifrName]");
                        String src = iframe.attr("abs:src");
                        if (src.contains("UpDocument")) {
                            //http://149.98.0.4/Information//UpDocument/20150624/201506241653353233.htm
                            src = src.replace("/Information/", "").replace("/information/", "").replace("\\", "/");
                        }

                        //重新添加爬取任务
                        //crawlDatums.add(src);
                        crawlDatums.add(new CrawlDatum(src)
                                .meta("title", title)
                                .meta("fbsj", fbsj)
                                .meta("bjsh", bjsh)
                                .meta("djcs", djcs)
                                .meta("firstUrl", firstUrl)
                                .meta("tqUrl", url)
                        );
                    } catch (Exception e) {
                        saveLog(title, fbsj, firstUrl, url, 0);
                        e.printStackTrace();
                    }
                }
            } else if (page.matchUrl("http://.*/Information/displaycont.asp\\?newsid=.*")) {
                //log.info("--------------------------displaycont:\n"+page.html());
                //保存爬取信息
                String title = page.meta("title") == null ? "" : page.meta("title");
                String fbsj = page.meta("fbsj") == null ? "" : page.meta("fbsj").replace(".", "-");
                String bjsh = page.meta("bjsh") == null ? "" : page.meta("bjsh");
                String djcs = page.meta("djcs") == null ? "" : page.meta("djcs");
                String firstUrl = page.meta("firstUrl") == null ? "" : page.meta("firstUrl");
                String tqUrl = page.meta("tqUrl") == null ? "" : page.meta("tqUrl");
                int zt = 0;

                try {
                    Date date = new Date();
                    String id = new CreateNewKey().createId();
                    TCollectorData tCollectorData = new TCollectorData();
                    tCollectorData.setId(id);
                    tCollectorData.setConfigId(tCollectorConfig.getId());
                    tCollectorData.setTitle(title);
                    Calendar nowdate = Calendar.getInstance();
                    String year = String.valueOf(nowdate.get(Calendar.YEAR));
                    if (null != fbsj && !"".equals(fbsj)) {
                        if (Integer.parseInt(fbsj.substring(0, 4)) > Integer.parseInt(year)) {
                            fbsj = year + fbsj.substring(4);
                            //log.info("fbsj:" + fbsj);
                        }
                    }
                    tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                    tCollectorData.setXxly("");
                    tCollectorData.setXxfb("");
                    tCollectorData.setBjsh(bjsh);
                    tCollectorData.setXxzz(bjsh);
                    tCollectorData.setDjcs(djcs);
                    tCollectorData.setWjgs("html");
                    tCollectorData.setUrl(tqUrl);
                    tCollectorData.setFydm(tCollectorConfig.getFydm());
                    tCollectorData.setFymc(tCollectorConfig.getFymc());
                    tCollectorData.setSystime(date);
                    collectorService.save(tCollectorData);

                    TCollectorContent tCollectorContent = new TCollectorContent();
                    tCollectorContent.setId(new CreateNewKey().createId());
                    tCollectorContent.setContent(html);
                    tCollectorContent.setText(HTMLSpirit.getTextFromTHML(html));
                    tCollectorContent.setSystime(date);
                    tCollectorContent.setDataId(id);
                    collectorService.save(tCollectorContent);

                    //遍历所有html中所有图片
                    Elements imgs = page.select("img[src]");
                    for (Element img : imgs) {
                        String src = img.attr("abs:src").replace("../", "");
                        log.info("img url:\n" + src);
                        crawlDatums.add(new CrawlDatum(src)
                                .meta("dataId", id)
                        );
                    }

                    //保存最大发布时间
                    TCollectorConfig config = (TCollectorConfig) collectorService.load(TCollectorConfig.class, " id='" + tCollectorConfig.getId() + "'");
                    if (config.getFbsj() == null || DateUtil.compareDate(fbsj, new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj())) > 0) {
                        config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        collectorService.save(config);
                    }

                    zt = 1;
                } catch (Exception e) {
                    zt = 0;
                    e.printStackTrace();
                } finally {
                    saveLog(title, fbsj, firstUrl, tqUrl, zt);
                }
            } else if (page.matchUrl("http://.*/UpDocument/.*.htm")) {
                //http://149.98.0.4/UpDocument/20180607/201806071428347285.htm
                //log.info("--------------------------http://.*/UpDocument/.*.htm");
                //保存爬取信息
                String title = page.meta("title") == null ? "" : page.meta("title");
                String fbsj = page.meta("fbsj") == null ? "" : page.meta("fbsj");
                String bjsh = page.meta("bjsh") == null ? "" : page.meta("bjsh");
                String djcs = page.meta("djcs") == null ? "" : page.meta("djcs");
                String firstUrl = page.meta("firstUrl") == null ? "" : page.meta("firstUrl");
                String tqUrl = page.meta("tqUrl") == null ? "" : page.meta("tqUrl");
                int zt = 0;

                try {
                    Date date = new Date();
                    String id = new CreateNewKey().createId();
                    TCollectorData tCollectorData = new TCollectorData();
                    tCollectorData.setId(id);
                    tCollectorData.setConfigId(tCollectorConfig.getId());
                    tCollectorData.setTitle(title);
                    tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                    tCollectorData.setXxly("");
                    tCollectorData.setXxfb("");
                    tCollectorData.setBjsh(bjsh);
                    tCollectorData.setXxzz(bjsh);
                    tCollectorData.setDjcs(djcs);
                    tCollectorData.setWjgs("html");
                    tCollectorData.setUrl(tqUrl);
                    tCollectorData.setFydm(tCollectorConfig.getFydm());
                    tCollectorData.setFymc(tCollectorConfig.getFymc());
                    tCollectorData.setSystime(date);
                    collectorService.save(tCollectorData);

                    TCollectorContent tCollectorContent = new TCollectorContent();
                    tCollectorContent.setId(new CreateNewKey().createId());
                    tCollectorContent.setContent(html);
                    tCollectorContent.setText(HTMLSpirit.getTextFromTHML(html));
                    tCollectorContent.setSystime(date);
                    tCollectorContent.setDataId(id);
                    collectorService.save(tCollectorContent);

                    //遍历所有html中所有图片

                    Elements imgs = page.select("img[src]");
                    if (null != imgs && imgs.size() > 0) {
                        for (Element img : imgs) {
                            String src = img.attr("abs:src").replace("../", "");
                            log.info("img url:\n" + src);
                            crawlDatums.add(new CrawlDatum(src)
                                    .meta("dataId", id)
                            );
                        }

                    }


                    //保存最大发布时间
                    TCollectorConfig config = (TCollectorConfig) collectorService.load(TCollectorConfig.class, " id='" + tCollectorConfig.getId() + "'");
                    if (config.getFbsj() == null || DateUtil.compareDate(fbsj, new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj())) > 0) {
                        config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        collectorService.save(config);
                    }

                    zt = 1;
                } catch (Exception e) {
                    zt = 0;
                    e.printStackTrace();
                } finally {
                    saveLog(title, fbsj, firstUrl, tqUrl, zt);
                }
            } else if (contentType != null && contentType.startsWith("image")) {
                String dataId = page.meta("dataId") == null ? "" : page.meta("dataId");
                String imageName = url.substring(url.lastIndexOf("/") + 1, url.length());
                //log.info("imageName:\n"+imageName);
                String serverFilePath = "";

                try {
                    serverFilePath = HttpClientUtils.getImage(url);
                } catch (IOException e) {
                    serverFilePath = "";
                    e.printStackTrace();
                } finally {
                    TCollectorElement tCollectorElement = new TCollectorElement();
                    tCollectorElement.setId(new CreateNewKey().createId());
                    tCollectorElement.setDataId(dataId);
                    tCollectorElement.setTitle(imageName);
                    tCollectorElement.setUrl(url);
                    tCollectorElement.setWjlj(serverFilePath);
                    tCollectorElement.setSystime(new Date());
                    collectorService.save(tCollectorElement);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void saveLog(String title, String fbsj, String firstUrl, String tqUrl, int zt) {
        try {
            log.info("savelog........." + fbsj);
            TCollectorLog tCollectorLog = new TCollectorLog();
            tCollectorLog.setId(new CreateNewKey().createId());
            tCollectorLog.setConfigId(tCollectorConfig.getId());
            tCollectorLog.setIp(tCollectorConfig.getIp());
            tCollectorLog.setModuleName(tCollectorConfig.getModuleName());
            tCollectorLog.setFydm(tCollectorConfig.getFydm());
            tCollectorLog.setFymc(tCollectorConfig.getFymc());
            tCollectorLog.setTitle(title);
            try {
                tCollectorLog.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
            } catch (Exception e) {
                e.printStackTrace();
            }
            tCollectorLog.setZt(zt);
            tCollectorLog.setFirstUrl(firstUrl);
            tCollectorLog.setTqUrl(tqUrl);
            tCollectorLog.setSysTime(new Date());

            collectorService.save(tCollectorLog);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
